package edu.hit.crawler;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.log4j.Logger;

import edu.hit.crawler.io.FetchOutput;

import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author
 */
public class StoreHtml {

	private static Logger logger = Logger.getLogger(StoreHtml.class);
	// private final static int MAX_SIZE_HTMLDIR = 100000;
	// private static int count = 0;

	private static String dir = "/home/zczhang/html/";

	// private static String tpDir = "";

	public static void WriteHtmltoLocal(String url, String domain, long date,
			byte[] content, String directory) {

		if (url.trim().length() == 0) {
			return;
		}

		FileOutputStream out = null;
		FileChannel fc = null;
		String fileName = dir + "(" + domain  + ").dat";
		try {

			File filetemp = new File(fileName);
			//System.out.println(filetemp);

			if (!filetemp.exists()) {
				filetemp.createNewFile();
			}

			// 获取源文件和目标文件的输入输出流
			out = new FileOutputStream(filetemp, true);
			// 获取输入输出通道
			fc = out.getChannel();
			String urlLine = "url: " + url + "\n";
			String dateLine = "date: " + date + "\n\n";
			fc.write(ByteBuffer.wrap(urlLine.getBytes()));
			fc.write(ByteBuffer.wrap(dateLine.getBytes()));
			fc.write(ByteBuffer.wrap(content));
			fc.write(ByteBuffer.wrap("\n\n".getBytes()));
			out.flush();
		} catch (Exception e) {
			e.printStackTrace();

		} finally {

			if (fc != null) {
				try {
					fc.close();
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}

			if (out != null) {
				try {
					out.close();
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}
		}
	}

	public static void main(String args[]) throws IOException {
		/*
		 * BufferedReader br = new BufferedReader(new FileReader("url"));
		 * Pattern p = Pattern.compile(
		 * "(?<=http://|\\.)[^.]*?\\.(com|cn|net|org|biz|info|cc|tv)",
		 * Pattern.CASE_INSENSITIVE); String line; while ((line = br.readLine())
		 * != null) { Matcher matcher = p.matcher(line);
		 * 
		 * if (matcher.find()) { String domain = matcher.group();
		 * myset.add(domain); System.out.println(domain); } }
		 */

		Configuration conf = new Configuration();
		System.out.println(conf.get("org.work.crawler.dir"));
		String workdir = conf.get("org.work.crawler.dir", "crawler/");
		String saveDir = conf.get("org.work.crawler.saveDir", dir);
		FileSystem fs = null;

		fs = FileSystem.get(conf);
		Path in = null;
		Path[] input;
		List<Path> list = new ArrayList<Path>();
		for (FileStatus stat : fs.listStatus(new Path(workdir + "doc/"))) {
			in = stat.getPath();
			list.add(in);
		}
		input = list.toArray(new Path[0]);
		for (Path path : input) {
			System.out.println(path);
		}

		for (FileStatus stat : fs.listStatus(input)) {

			in = stat.getPath();

			SequenceFile.Reader reader = null;
			if (in.getName().startsWith("_"))
				continue;

			System.out.println("process the file : " + in);
			try {
				reader = new SequenceFile.Reader(FileSystem.get(conf), in, conf);
				edu.hit.crawler.io.CrawItem key = new edu.hit.crawler.io.CrawItem();
				FetchOutput value = new FetchOutput();
				Pattern p = null;
				while (reader.next(key, value)) {
	/*				p = Pattern
							.compile(
									"(?<=http://|\\.)[^.]*?\\.(com|cn|net|org|biz|info|cc|tv)",
									Pattern.CASE_INSENSITIVE);
					Matcher matcher = p.matcher(value.getUrl());
					String domain = null;
					if (matcher.find())
						domain = matcher.group();
					else
						domain = new URL(value.getUrl()).getHost();
*/
					String domain = new URL(value.getUrl()).getHost();
					if (value.getCode() == FetchOutput.SUCCESS
							&& value.getContent().length > 200)
						WriteHtmltoLocal(value.getUrl(), domain,
								value.getLastModifyTime(), value.getContent(),
								saveDir);
				}

				reader.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
	}
}
